package util.kwdDigg;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import util.HtmlUnit.SpiderContentTxt;

/*这个类作用是处理采集下的网页内容，使用了正则表达式截取出需要采集的网址*/

public class SpiderRobotKWD {

	public void run() {

		Connection conn = null;
		Statement stmt = null;
		Connection connCheck = null;
		Statement stmtCheck = null;
		Connection connGetSite = null;
		Statement stmtGetSite = null;
		java.sql.ResultSet rs = null;
		java.sql.ResultSet rsGetSite = null;
		SpiderContentTxt scx = new SpiderContentTxt();
		// workingID是个标记位，指示在现在处理到哪一位了
		int workingID = 0;
		boolean alltimes = true;
		try {
			Class.forName("com.mysql.jdbc.Driver");
		} catch (ClassNotFoundException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		try {
			conn = DriverManager.getConnection("jdbc:mysql://" + host + "/" + dBName + "?user=root&password=777888999a!&useUnicode=true&characterEncoding=utf8");
			connGetSite = conn;
			connCheck = conn;
			stmtGetSite = connGetSite.createStatement();
			stmtCheck = connCheck.createStatement();
			SubRobot sr = new SubRobot();
			sr.setdBName(dBName);
			sr.setHost(host);
			sr.AddParameter();
			// 查询出需要处理的网址，check是标记为0就是需要处理的
			while (alltimes) {
				rsGetSite = stmtGetSite.executeQuery("select * from keywordnofilter where spider = 0 limit 8;");
				if (!rsGetSite.next()) {
					alltimes = false;
				} else
					do {
						long btime = System.currentTimeMillis();
						workingID = rsGetSite.getInt("id");
						// 抓取网址开始的时间点
						// System.out.println(first + keyword + end);
						int reConnectTime = 5;
						String SitesString = null;
						while (reConnectTime > 0) {
							SitesString = scx.SpiderPageContent(first + URLEncoder.encode(rsGetSite.getString("keyword"), "utf-8") + end);
							if (SitesString.length() > 10) {
								reConnectTime = reConnectTime - 5;
							} else {
								reConnectTime--;
							}
						}
						// 修改标记放在处理抓取下来的字符串之前目的就是如果这条数据出错了，下次在重启时也不需要再处理了，虽然少处理一条数据但保证了程序的连续性
						stmt = conn.createStatement();
						stmtCheck.execute("update keywordnofilter set spider = 1 where id = " + workingID);
						Pattern areaPattern = Pattern.compile(area);
						Matcher areaMatcher = areaPattern.matcher(SitesString);
						areaMatcher.matches();
						areaMatcher.reset();
						if (areaMatcher.find()) {
							Pattern p = Pattern.compile(urlPattern);
							Matcher m = p.matcher(areaMatcher.group());
							int tagURLController = 0;
							while (m.find()) {
								String newWord = m.group(2);
								// 查出的关键字必须包含核心关键词
								newWord = newWord.replaceAll("<strong>", "").replaceAll("</strong>", "").replaceAll("</b>", "").replaceAll("<b>", "").replaceAll("\\n*", "").trim().replaceAll("\\s{1,200}", "+");
								//System.out.println(newWord);
								sr.setSubId(rsGetSite.getInt("subId"));
								if (!sr.getSubNo(newWord)) {
									continue;
								}
								rs = stmt.executeQuery("select id from keywordnofilter where keyword = '" + newWord + "'");
								int wordRank = 0;
								if (rsGetSite.getInt("wordRank") > 0) {
									wordRank = rsGetSite.getInt("wordRank") - 1;
								}
								if (!rs.next()) {
									System.out.println(newWord);
									stmt.execute("insert into keywordnofilter (keyword,subid,wordRank) values ('" + newWord + "'," + rsGetSite.getInt("subId") + "," + wordRank + ")");
								} else {
									System.out.println("kwd chongfu");
								}
								tagURLController++;
							}

							Long etime = System.currentTimeMillis(); // 打印出处理使用的时间
							System.out.println(dBName + "--" + workingID + "---" + rsGetSite.getString("keyword") + "----使用了" + (etime - btime) + "毫秒----" + new Date());
							// System.out.println(dBName + "----" +
							// workingID + "---" + tag + "----使用了" + (etime
							// - btime) + "毫秒----" + new Date());
						}
					} while (rsGetSite.next());
			}
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} finally {
			try {
				if (rs != null) {
					rs.close();
					rs = null;
				}
				if (stmt != null) {
					stmt.close();
					stmt = null;
				}
				if (conn != null) {
					conn.close();
					conn = null;
				}
				if (connCheck != null) {
					connCheck.close();
					connCheck = null;
				}
				if (stmtCheck != null) {
					stmtCheck.close();
					stmtCheck = null;
				}
				if (connGetSite != null) {
					connGetSite.close();
					connGetSite = null;
				}
				if (stmtGetSite != null) {
					stmtGetSite.close();
					stmtGetSite = null;
				}
				if (rsGetSite != null) {
					rsGetSite.close();
					rsGetSite = null;
				}
				if (scx != null) {
					scx = null;
				}
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
	}

	String urlPattern = null;
	String host = null;
	String dBName = null;
	String first = null;
	String area = null;
	String proxyString = null;
	String end = null;
	String cookiesString = null;

	public void setArea(String area) {
		this.area = area;
		// System.out.println(this.area +1);
	}

	public void setFirst(String first) {
		this.first = first;
	}

	public void setEnd(String end) {
		this.end = end;
	}

	public void setHost(String host) {
		this.host = host;
	}

	public void setdBName(String dBName) {
		this.dBName = dBName;
	}

	public void setUrlPattern(String urlPattern) {
		this.urlPattern = urlPattern;
	}

	public void setCookiesString(String cookiesString) {
		this.cookiesString = cookiesString;
	}

	public void setProxyString(String proxyString) {
		this.proxyString = proxyString;
	}

}
